{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "id": "aenHhwSWhMGu",
    "colab_type": "code",
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 393.0
    },
    "outputId": "7296d0cd-fd1d-4ff7-9312-57d52b05e64d"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 001000 cost = 0.001100\n",
      "Epoch: 002000 cost = 0.000286\n",
      "Epoch: 003000 cost = 0.000123\n",
      "Epoch: 004000 cost = 0.000062\n",
      "Epoch: 005000 cost = 0.000034\n",
      "sorry hate you is Bad Mean...\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAJkAAAEMCAYAAADXgBT+AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3XlUVdUeB/DvBQREzUeOTM4C5uPK\nRQzIAbNwIEeQFMQizCkHJLW0Z+HTVMrKFEvp6RPLwhic0kyhVByAGBQxRwYVSCC4gDJP+/3h4y5u\n14t3OJvJ32ct14LD2ft3Lv44496/I2KMMRDCkU5LbwBp/yjJCHeUZIQ7SjLCHSUZ4Y6SjHBHSUa4\noyQj3FGSEe70WnoDyGN+fn4qr7t9+3aOWyI82pO1EkZGRrJ/BgYGiImJQXZ2Njp06AA9PT3cvXsX\n58+fx/PPP9/Sm6o2ET27bH0CAgJgZ2eHadOmyS2PiIhASkoKNm7c2EJbphlKslbI3t4ecXFx0NOT\nP5upqamBk5MTEhMTW2jLNEOHy1aoa9euOHPmjMLymJgYdOnSpQW2SDt04t8KLVq0CMuXL4eVlRXM\nzc1RW1uL3Nxc3Lp1C+vWrWvpzVMbHS5bqczMTERHRyMvLw/V1dXo2bMnxowZA7FY3NKbpjZKslZo\n586dWLp0aUtvhmDonKwVCgsLg1QqbenNEAztyVqhkJAQnD59Gq6urjAxMVG4ynR2dm6hLdMMJVkr\nZG1trfRnIpEIN27caMat0R4lGeGObmG0UowxXL16FTk5OQCAfv364YUXXmjhrdIMJVkrlJaWhkWL\nFiE7OxudOnUCAJSVlcHKygp79uxBjx49WngL1UOHy1borbfegqmpKVauXCl7IJ6Xl4etW7eipqam\nzY3CoCRrhSQSCWJjY2FoaCi3vLS0FBMmTMDFixdbaMs0Q/fJWqEuXbqgvLxcYXlNTQ1EIlELbJF2\n6JysFRo5ciRWrFgBf39/DBw4EACQnp6O7du3Y8SIEYLEqKioUHndjh07ahWLDpet0KNHj/Cvf/0L\nUVFRsmWMMTg7OyMwMBDGxsZax7C2tlZ5r6jtfTlKslaouroa+vr6ePjwIXJyclBdXQ0LCwtBR8X+\n/vvvsq8zMjIQGhqK119/Hf3790d9fT3S0tJw6NAhzJs3T2HwpNoYaXVsbGzYm2++yXbv3s1SUlJY\nfX0913gzZ85k9+7dU1ielpbG3NzctO6fzslaobCwMCQlJSExMREHDhxAVVUVXnzxRTg5OcHJyQkD\nBgwQNF5aWhp69eqlsNzU1BQZGRla90+HyyakpaWpvO6gQYO4bUdWVhYuXLiA77//Hunp6YI/u/T0\n9ISJiQkWLFggN0hyz549uH//PsLCwrTqn5KsCQ0nx4wxuZPkv38PaH9y/HclJSVISkqS7dEyMzNh\nZWUFOzs7+Pv7Cxrr3r17eP/993HlyhXZ52KMYdCgQdi+fbvsCldTlGRNaHhuCADJycmIjIyEj4+P\n3Mnxd999hzfeeAOvvvqqYHFfe+01VFVVQSwWw87ODra2thgyZAh0dXUFi/EkBQUFciNxzczMBOmX\nzsma0PiXvHDhQoSEhKB79+6yZf3794eNjQ18fX0FTTITExNcv34dDx48QH5+PgoLC1FeXs5tEsmE\nCRNw6tQpdO/eXe7zCYXu+Kvozz//hIGBgcJyIyMj5ObmChprz549uHDhAj744AP84x//wMGDB+Hi\n4oJp06ZxmXNpYmLyxNlRQqHDpYrefvttVFZW4q233oKZmRnq6urw4MEDfPvtt+jQoQP27t3LJW51\ndTVSU1ORmJiII0eO4N69e7h+/bqgMdauXYszZ87A1NQUpqamCodlbR/I0+FSRZ9++ik2bdqElStX\norKyEgCgp6cHJycnbNq0SdBY0dHRuHz5MpKTk/HHH3/AxMQEjo6OWLZsGRwdHQWN1eDll1/m0i9A\nezKVNb6iLC4uRnV1NZ5//nmF8fdCePnll+Hg4ABHR0c4Ojqid+/eT1zv9u3bsLS0FDy+0CjJVCSR\nSHD58uWW3gw5w4YNQ0pKiiB9xcfH4+eff0ZOTg5EIhH69u2L6dOn45///KfWfdOJv4peeeUV/PDD\nDy29GXKE2j/8+OOPmDdvHh48eIB+/fqhb9++yMzMxOzZsxETE6N1/3ROpqLi4mJs374dQUFB6N27\nt8LJcURERLNvk1Bjy/773/9i586dGDt2rNzyqKgofPnllxgzZoxW/VOSqcjW1ha2trYtvRlc5Obm\nPjGRxo0bhzVr1mjdPyWZitpT2YC/Mzc3R1JSksKAyJSUFEEmrVCSqeHQoUNyJ8d9+vSBu7s7XFxc\nWnrTtOLj44MFCxZg8uTJsueUGRkZOH78OJYvX651/5RkKvr666+xf/9+uLq6YtSoUQAe/0esWbMG\nZWVlmD59egtvoeY8PDzQo0cPRERE4PDhw6iurkafPn2wefNmTJw4UfsAWo9Ie0aMHTuWXb58WWF5\nQkICmzx5cgtsEWP29vaC9BMZGckePHggSF9PQvfJVCSRSJCYmKhwVVlXV4cRI0YgOTlZq/7PnTun\n8rpCF1yZMWMGbt++DXNzc7z00ktwcnKCo6MjnnvuOUH6pyRT0YwZM7B48WKMHz9ebnl0dDR27NiB\nY8eOadX/34usNIxja0xHRwcdO3ZEUlKSVrGepLS0FElJSUhOTkZiYiKuX7+OAQMGYOTIkXj33Xe1\n6puSTEW//fYbli1bBgcHB7mT4/j4eGzZsgVTpkwRLFZ0dDROnjyJRYsWyY1d++abb+Dq6qqQ6ELL\nyMhAbGwsQkNDBRmJS0mmhjt37iAiIgJZWVmyk+Pp06cLXmLTxcUFhw8fRufOneWWFxcXw93dHb/+\n+qug8VJTU2UjcK9cuQJjY2NIJBJIJBLY2dmhb9++WvVPV5cqCgoKwksvvYTVq1dzeSjeWHFxMcrK\nyhSSrKqqCiUlJYLH8/DwwIABA+Dl5YXAwECFuNqiPZmKli5diqSkJFRVVcHe3l42c6ipgnWaWrly\nJW7cuIFZs2bB3NxcNnYtPDwcAwcOFLzgys8//4yEhAQkJCSgpKQEdnZ2sLe3h729vVqTgJWhJFNT\nenq67NCSlJSEiooKODk54fPPPxcsRkVFBXbt2oXo6Gjk5ubKVb9etWqV4HuaxoqKipCYmIj4+Hhc\nuHABhYWFSEhI0KpPSjINlJWVITk5GUlJSYiOjsbdu3dx7dq1lt4srd27dw/Jycmyz5afn4/hw4cj\nODhYq34pyVT0yy+/yKao5eXlwcbGRnZyLBaLFco8aYMxhrNnzyI9PV02CrcxoZ+jvvPOO0hJSUFZ\nWRlsbW1lgyVtbGwEmSFFSaYia2tr2cmxm5sbjIyMuMVavXo1Tp48ib59+yokr0gkEnxY0fbt2+Ho\n6AiJRAJ9fX2l623duhWrV69Wu39KMhVdvXoVv//+OxISEpCSkgJzc3PZybGdnZ2gxVCGDx+O77//\nnstFhTY0HYlLSaYBxhhu3LiB2NhYRERE4O7du4LOIB83bhx+/vlnQQ/BQhCLxbh69ara7eg+mRrK\nysqQkpIiOzm+cuUKevfuDU9PT0Hj+Pn54bPPPsOKFSu4XkmqS9NbGbQnU9G0adOQlpYGExMTuZlE\nPCpRT506FX/++SfKysrw3HPPQUdHfipGbGys4DFVoenhkvZkKpo7dy4cHR1hbm7e5Hr+/v7Ytm2b\nVrHeeustrdq3NrQnE5iQ09RaG9qTtRJC/M3W1tZi165dTxzq7ePjo/1GakjTz0ZJJjAhpql98skn\n+O233+Dp6SkbAZGeno59+/ahrq4O8+bN0zqGJl5//XWN2tHhUmBCHC5HjRqF/fv3KxSfu3nzJvz8\n/HDq1Cmt+gceX8GqStsH8jSDvBWqqKhAnz59FJYPGjQIhYWFgsQwMjKS/TMwMEBMTAyys7PRoUMH\n6Onp4e7duzh//rwgN5npcNkKDR48GKGhoXjjjTfklh88eBD9+/cXJMaWLVtkXwcEBGD9+vUKpdQj\nIiIEuYihw6XANL0r3tjly5fh6+uLnj17yg31zs3NxVdffYWRI0cKsaky9vb2iIuLUxiMWVNTAycn\nJyQmJmrVPx0uVbRjx44nLi8rK8OGDRtk369YsULrWBKJBL/++ivmzJkDMzMz9OjRAx4eHjh58qTg\nCQYAXbt2fWKlxZiYGEFKiNKe7CmkUikKCwvh7u6Ow4cPK1zG3717F++++67We6+/y8vLg66urqyG\na0ZGBgwNDWFqaipoHAAIDw/HRx99BCsrK7kS67du3cK6deswZ84crfqnJHuKw4cPY8uWLXj06JHS\n+0Tjx49XuqfTxLlz5+Dn54fAwEDZDO4ff/wRgYGB2LFjB0aPHi1YrAaZmZmIjo6Wq349ZswYYSbJ\ncJs23I7U1tYyW1tblp2drfCvoKBA8HhTpkxhx48fV1geFRXFpk6dKni8oKAgwftsjJJMDXV1dXJf\n//HHH0wqlQoex9bWltXW1iosr66uZra2toLHGz16NCssLBS83wZ0C0NFcXFxeO+99xATE4Pa2lp4\ne3vjypUr0NfXR1BQkKClA/r164dTp07B1dVVbnlERMRTH9BrwtfXF0uXLoWrqytMTEwUrjK1/Wx0\nTqYid3d3zJ49Gx4eHjh69Ci2bt2KsLAwXLlyBXv37kVkZKRgsS5evIilS5fC3NwcZmZmYIwhMzMT\n+fn52LdvHyQSiWCxAMUSCY2JRCKaQd5c7OzskJSUBJFIBH9/f/To0QMffPABGGOwt7cXvD5FXl4e\nTp48iaysLIhEIlhYWGDy5Mno1q2boHGaAx0uVWRoaIiHDx/CwMAAly5dwpdffgngcaESHu8F79Wr\nF2bPno38/PwnPmLSVmVlpWx499NeFa3t66EpyVTk7OyMN998E7q6ujA2NoajoyOqqqqwadMmDB8+\nXNBYxcXF2LhxI3755ReIRCJcu3YNUqkUfn5++Pzzz9GzZ0+tYzg4OMgeGUkkkif+obD/v7tA28Ml\nJZmKAgICEBISgkePHsHLywsikQj19fX466+/sHnzZkFjbdiwAWVlZQgLC5PNHzAyMoKFhQU+/vhj\nQe7JNX5Nz7fffqt1f03idt3azvC+l9SYvb09KyoqYowxJhaLZcsfPXrEHBwcmm07hEJ7MhWFhYXB\ny8tL0PmVyujp6T1xOlx1dTWqqqoEj3fz5k1s3boV6enpT+xf24krlGQq4n0vqTGJRIJPPvkEq1at\nki27f/8+Pv74Yzg5OQkWp8GKFStgZmaGhQsXcpnrSbcwVMT7XlJjubm5WLx4MW7fvo26ujoYGhrK\nSlZt3bpV6Qu9NDVs2DDExcVpfRWpDO3JVHTz5s1mi9W7d298+umnqKysRFZWFgoKClBeXg5LS0vB\nEwx4PJ4sPz9f64qKytCerAnNeS+psQMHDiAoKAjx8fEoLi6Gq6srunTpguLiYixYsECQiSSNq20X\nFBQgLCwMU6dOhZmZmcLtDHqsxFHjSSHKKg4yge4lNebi4oIvvvgCNjY22LdvHw4dOoQjR44gIyMD\nS5YswenTp7WO0fjw/6RK241/RvfJOGp8pdW9e3ds27ZNsNf/NaWgoAA2NjYAgAsXLsDV1RW6uroY\nPHgw/vrrL0FiND78z507F999953COg33BLVFSdYEExMTLFu2DBYWFiguLm7yZfAvvviiYHGNjY2R\nlpYGQ0NDxMfH47333gMA5OTkCFoXLTU1FVevXsXly5fxww8/KPwBZWVlITs7W+s4lGRN+PTTT7F/\n/35cu3YNdXV1SE1NfeJ6Qj+79PT0hLu7O0QiEUaNGgUrKys8evQIS5YsEeZdR/9XUVGB8+fPo7a2\nFnv27FH4uaGhoVrzM5VqqbvAbY23t3ezxktOTmZnz55lVVVVjLHHo3ODg4NZdXW14LHmz58veJ+N\n0Yk/4Y6mxBHuKMkId5RkhDtKMsIdJRnh7pm/T3b27mCN2o0wPYGEP19Tq03gpJkaxdp9zA+Lpqpf\nI6w+477abb5JDsQCuzVqtztd9b3Sn9GeTEOd9C2bLVa/wcKPvFAaa6iF4H1SkhHuKMkId5RkhDtK\nMsIdJRnhjpKMcEdJRrh7apJlZ2fDysoKt2/f5rohhw4dgoODA9cYpGVw35OFhISgurqadxgAjwuV\nLF26FFZWVoIMGybC4JpkUqkUgYGBqKmp4RkGAJCWloYZM2YIUhKcCEvlJLt+/TqmTp0KiUQCb29v\n5OTkAABOnDiBKVOmQCKRwNnZGbt37wbwuIjbmDFjwBiDo6MjwsPDAQDHjh3DxIkTYWtrCw8PD4U3\nXpw5cwYuLi6wsbGBv7+/ynvBgoICBAYGtrt3RbYLTxufnZWVxSwtLdmsWbNYdnY2KykpYT4+PmzO\nnDksKyuLDRkyhJ05c4YxxlhKSgobOnQou3DhAmOMsbi4OGZpaclKS0sZY4ylpqYysVjMLl26xGpq\nalhwcDBzcHBgFRUVLDIykonFYrZp0yZWVlbGbty4wWxsbNixY8fUGk9+69YtZmlpybKyslRav7Tq\nllr9E/WpPArD09MTZmZmAB4XH5k/fz6MjY0RGxuLrl27Anj8ypf+/fvj2rVrT3xzxpEjR+Dg4CAr\nGuLj4wNTU1PZ4bSqqgrLli2DkZERrK2tYWVlhbS0NK3/kJqi7kiKBmP73VF7BIemozB+ubEFE4es\nVbudJqMwTld9j/EG6r8coqlRGCon2aBBg2Rf9+nTB4wx5Ofn49SpU4iMjEReXh4YY6ipqVF6iMvK\nypKr3qyvr4/JkyfLvu/atavcOZWhoWGzXTQQflROssYvW2f/n+B04sQJ7N27F0FBQXB0dISenh6m\nT5+utI+G6oTk2aLyiX9mZqbs6/v370NXVxc5OTmws7PDqFGjoKenh9LSUty7d09pHxYWFnL91NfX\nY9++fcjLy9Nw80lboHKShYaGIj8/H6Wlpdi/fz+cnZ3Rv39/ZGZmoqioCLm5ufjwww9hYmIiS5qG\nijiZmZkoLy+Hm5sbkpKSEB0djZqaGhw4cADBwcHo3Lkzn09HWgW1Tvx9fX2Rk5MDsViMzZs3w8jI\nCAkJCRg3bhx69uyJtWvXQiqVYsOGDejWrRuWLFkCOzs7zJ49G8uXL8eCBQvw5ZdfIjAwEKtWrYKl\npSWCg4PRqVMnrT/IunXrcPToUdmhfOLEiRCJRNi4cWOTh3DC3zM/g1zTMf50danYThl6QE64axOz\nlRYtWoSLFy8q/fn7778Pb2/vZtwioo42kWQNj6pI20SHS8IdJRnhrk0cLnkqrtO8PKbabXW0qMio\nQVuRoYFGoTRtpwztyQh3lGSEO0oywh0lGeGOkoxwR0lGuKMkI9xRkhHuKMkId1SmgHDXbsoUVFZW\nYsuWLXB2dsbw4cPh5eWFK1eucI9Lnq7dlCnYunUrYmNjceDAAVy6dAnDhg3D4sWLUVlZyT02aVq7\nKVPQuXNnrF27FhYWFjAwMICvry+kUinS09NV/YiEl6dNMW9rZQoapKSkMGtra5aXl9fkeiVVdzTq\nn6iuXZYpKC8vR0BAADw8PNCzZ88m1/0ty13t/gFg+sAUHEkfplab3VM1K4nwyx+bMXHoB2q3Y9m5\narc5VfJfTOjqq1E7ZdpdmQKpVIqFCxeiW7duWLdunVptCR8qn5MpK1MQHByMgIAAJCcnIzU1FdbW\n1kr74F2mIDs7G7NmzcLAgQOxa9cu6Ovrc4tFVNduyhQUFRVh3rx5GD9+PAIDA9GhQwet+yTCaDdl\nCrZt24bBgwdj9erVWvdFhNVuyhSEh4dDV1cXNjY2csupTEHLe+bLFKh7hdiAri4V2ylDD8gJd21i\nShyVKWjb2kSSUZmCto0Ol4Q7SjLCHSUZ4a5NnJPx1E23tEXaNof6svJmbacM7ckId5RkhDtKMsId\nJRnhjpKMcEdJRrijJCPcUZIR7qhMAeGu3ZQpyM/Ph7+/PxwcHDBixAh4e3vj6tWr3OOSp2s3ZQr8\n/Pygq6uLqKgonD9/HkOHDsXChQubJTZpWrsoU1BfX4+ZM2dizZo1eO6552BoaIgZM2ZAKpUiPz9f\n5V8G4UPlJDt48CB27dqFc+fOoUOHDnj//feRnZ2N1atXY+XKlbh8+TKCgoKwc+dOXLx4Eb169cLe\nvXsBAHFxcfDw8MC1a9fw4YcfIiAgAImJiXBxccHChQtlRVEqKysRGxuLo0ePIjw8HL/++itOnTr1\n9A+howN3d3d0794dAFBYWIh9+/ZBIpHA1NRUk98LEdLT6hg01MI4cuSIbFlMTAyzsrJipaWlrLi4\nWG79yZMns927dzPGFGthbNy4kc2fP1+2blVVFfvpp5/Yw4cPWWRkJLOysmIPHz6U/XzmzJnsiy++\nUKvugpOTE7O0tGTe3t4sPz//qeuXVt1Sq3+ivnZXpuDSpUsoKCjA7t274enpiaNHjzY55S75z0lq\n9d9gdL90nL87UK02m16bpVEsTWcr1d3KULtNVO1BuOjN1qidMu2qTEGD7t27Y82aNSgsLMTZs2e5\nxyNNaxdlCnJycjB27FhkZWXJluno6IAxBj29Z35cZotrF2UKTE1N0a1bN2zevBlSqRSVlZXYvn07\n9PX1YW9vr1XfRHvtokyBSCTC119/jS1btuDVV1+Fjo4OrK2t8c0336Bbt25a9U2098yXKVD35L0B\nnfgrtlOGHpAT7trEWTGVKWjb2kSSUZmCto0Ol4Q7SjLCHSUZ4a5NnJPxtGGAnUbtourVb6vTRf3K\nhw00qpqYnaRRLE3bKUN7MsIdJRnhjpKMcEdJRrijJCPcUZIR7ijJCHeUZIQ7KlNAuGs3ZQoai46O\nhpWVFeLj45s1LnmydlOmoEF5eTkCAwNhZGTUbDFJ09pFmYLGgoKC4OTkBGNjY7XaEX7aRZmCBrdu\n3cJPP/2Ed999V81fA+FJrdlKZmZmAABfX1/Mnz8fxsbGiI2NRdeuXQEAYrEY/fv3x7Vr1zBy5EiF\nPo4cOQIHBwc4OTkBAHx8fGBqaio7nFZVVWHZsmUwMjKCtbU1rKyskJaWptL2McYQEBAAf39/tfZi\n31z9HP3/2Ufl9RuLqg/XqJ0mmnqfpNB0et8RtL92U6ag4c29bm5uqn4kAMAC8Uq11m8QVR8OFx0P\ntdroNPps6tD0Racnb51Xu41O7zuozx2sUTtlVE4yZWUK9u7di6CgIDg6OkJPT6/JVzHzKlMglUqx\nY8cOhISEQCQSCd4/0U67KFNw7tw5FBUVYc6cOXBwcICDgwMePHiAd955Bxs3btSqb6I9lfdkoaGh\nsLe3h5GRkVyZgvj4eBQVFaGqqgqffPKJ0jIFAwYMgJubGzw8PBAdHQ1nZ2eEhoYiODgYr7/+ulYf\nYuLEibLzvAazZs3CmjVr8NJLL2nVN9FeuyhT0LFjR3Ts2FFuma6uLp5//nnZRQlpOc98mQJ1T94b\n0Im/YjulP1O7N0LU1CZmK1GZgratTSQZlSlo2+hwSbijJCPcUZIR7trEORlPog76zda29NUXNI6l\nSdt3ctQfx7e7N/BOjqNG7ZShPRnhjpKMcEdJRrijJCPcUZIR7ijJCHeUZIQ7SjLCHSUZ4Y5qYRDu\nuD9WCgkJgZeXF/T1NX98o4q5c+ciKSkJurq6smV9+vTBiRMnuMYlT8c1yRpqYXh4eHBPMgBYvHgx\nli1bxj0OUU+7q4VBWiH2FFlZWczS0pLNmjWLZWdns5KSEubj48PmzJnDsrKy2JAhQ9iZM2cYY4yl\npKSwoUOHsgsXLjDGGIuLi2OWlpastLSUMcZYamoqE4vF7NKlS6ympoYFBwczBwcHVlFRwSIjI5lY\nLGabNm1iZWVl7MaNG8zGxoYdO3bsaZvIGGPM29ubeXl5sSlTpjBbW1vm5eXF0tLSntou89p9lfon\nmms3tTAGDhyI+vp6+Pn5QU9PD5s2bcL8+fNx8uRJGBgYKG23wG6Nqr8COaervsd4gzlqtSmbLNEo\n1sXIVRjp/pna7Ww+SHn6Sn+ze/h3WJQ0V6N2yrSbWhjr16+X+/6jjz6Cg4MDfv/9d4wePVqlPggf\n7aIWxpN07twZXbt2RX5+frPEI8q1i1oYpaWlWL9+vVw/UqkUUqkUFhYWWvVNtKdykoWGhiI/Px+l\npaVytTAyMzNRVFSE3NxcfPjhh0prYZSXl8PNzQ1JSUmIjo5GTU0NDhw4gODgYHTu3FmrD9G5c2ek\npKTg448/RnFxMUpKSvDvf/8blpaWsLe316pvoj2Vk6yhFsbo0aNRW1uL9evXw9PTEwMHDsS4cePw\n5ptvYtq0aXj77bdx/PhxbNu2DUOGDJHVwjhw4ACGDBkiq4UxYsQIHD9+XJBaGADw1VdfgTGGCRMm\nYOzYsaipqcF//vMfucM8aRnPfC0Mda8QG9DVpWI7ZejPnHDXJqbEUS2Mtq1NJBnVwmjb6HBJuKMk\nI9xRkhHu2sQ5GU+srq7Z2uqX1GocS5O2gzpq9khN03bK0J6McEdJRrijJCPcUZIR7ijJCHeUZIQ7\nSjLCHSUZ4Y7KFBDuuO/JQkJCmm2CbkxMDKZMmQKxWIxJkybh9OnTzRKXNI1rkjWUKWiYV8nTzZs3\nsXLlSqxYsQIJCQlYvnw5vvrqK5SVlXGPTZrWbsoU7N+/H5MmTcIrr7wCAwMDTJo0CUePHhVk/gDR\n0tOmmLeVMgUuLi4sKCiI+fj4MIlEwtzc3FhCQsJT22WkUpkC3tpNmYLc3FyEh4djx44dGDRoEIKD\ng7Fo0SJERUXB2NhYabsFtqtV/RXIiao9CBe92Wq1qR1rq1GsM9Fr8PKrgWq3e23HGbXbrHrhFD67\nPkGjdsqofLhUVqYgNDQULi4uEIvFsLGxwZ07d9QuU9BQmkCbMgWMMUyZMgXDhg1Dp06dsHz5cujo\n6CAmJkbVj0g4aTdlCnr06CH3vnE9PT306tWLyhS0Au2iTAHweE9748YN2fe1tbXIzc2VHeJJy2kX\nZQoAwNvbG6dPn0Z0dDQqKyuxc+dO6OvrY+zYsVr3TbSj1om/r68vcnJyIBaLsXnzZhgZGSEhIQHj\nxo1Dz549sXbtWkilUmzYsAHdunXDkiVLZGUKli9fjgULFsjKFKxatQqWlpaClSkYM2YMAgICsHnz\nZvz111+wtrbG3r17YWRkpHXchSXpAAAAVElEQVTfRDvPfJkCda8QG9DVpWI7ZegBOeGuTcxWojIF\nbVubSDIqU9C20eGScEdJRrijJCPcPfO3MAh/tCcj3FGSEe4oyQh3lGSEO0oywh0lGeHufxMp6FYg\nfNQPAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x216 with 1 Axes>"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "'''\n",
    "  code by Tae Hwan Jung(Jeff Jung) @graykode\n",
    "  Reference : https://github.com/prakashpandey9/Text-Classification-Pytorch/blob/master/models/LSTM_Attn.py\n",
    "'''\n",
    "import tensorflow as tf\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "\n",
    "tf.reset_default_graph()\n",
    "\n",
    "# Bi-LSTM(Attention) Parameters\n",
    "embedding_dim = 2\n",
    "n_hidden = 5 # number of hidden units in one cell\n",
    "n_step = 3 # all sentence is consist of 3 words\n",
    "n_class = 2  # 0 or 1\n",
    "\n",
    "# 3 words sentences (=sequence_length is 3)\n",
    "sentences = [\"i love you\", \"he loves me\", \"she likes baseball\", \"i hate you\", \"sorry for that\", \"this is awful\"]\n",
    "labels = [1, 1, 1, 0, 0, 0]  # 1 is good, 0 is not good.\n",
    "\n",
    "word_list = \" \".join(sentences).split()\n",
    "word_list = list(set(word_list))\n",
    "word_dict = {w: i for i, w in enumerate(word_list)}\n",
    "vocab_size = len(word_dict)\n",
    "\n",
    "input_batch = []\n",
    "for sen in sentences:\n",
    "    input_batch.append(np.asarray([word_dict[n] for n in sen.split()]))\n",
    "\n",
    "target_batch = []\n",
    "for out in labels:\n",
    "    target_batch.append(np.eye(n_class)[out]) # ONE-HOT : To using Tensor Softmax Loss function\n",
    "\n",
    "# LSTM Model\n",
    "X = tf.placeholder(tf.int32, [None, n_step])\n",
    "Y = tf.placeholder(tf.int32, [None, n_class])\n",
    "out = tf.Variable(tf.random_normal([n_hidden * 2, n_class]))\n",
    "\n",
    "embedding = tf.Variable(tf.random_uniform([vocab_size, embedding_dim]))\n",
    "input = tf.nn.embedding_lookup(embedding, X) # [batch_size, len_seq, embedding_dim]\n",
    "\n",
    "lstm_fw_cell = tf.nn.rnn_cell.LSTMCell(n_hidden)\n",
    "lstm_bw_cell = tf.nn.rnn_cell.LSTMCell(n_hidden)\n",
    "\n",
    "# output : [batch_size, len_seq, n_hidden], states : [batch_size, n_hidden]\n",
    "output, final_state = tf.nn.bidirectional_dynamic_rnn(lstm_fw_cell,lstm_bw_cell, input, dtype=tf.float32)\n",
    "\n",
    "# Attention\n",
    "output = tf.concat([output[0], output[1]], 2)                             # output[0] : lstm_fw, output[1] : lstm_bw\n",
    "final_hidden_state = tf.concat([final_state[1][0], final_state[1][1]], 1) # final_hidden_state : [batch_size, n_hidden * num_directions(=2)]\n",
    "final_hidden_state = tf.expand_dims(final_hidden_state, 2)                # final_hidden_state : [batch_size, n_hidden * num_directions(=2), 1]\n",
    "\n",
    "attn_weights = tf.squeeze(tf.matmul(output, final_hidden_state), 2) # attn_weights : [batch_size, n_step]\n",
    "soft_attn_weights = tf.nn.softmax(attn_weights, 1)\n",
    "context = tf.matmul(tf.transpose(output, [0, 2, 1]), tf.expand_dims(soft_attn_weights, 2)) # context : [batch_size, n_hidden * num_directions(=2), 1]\n",
    "context = tf.squeeze(context, 2) # [batch_size, n_hidden * num_directions(=2)]\n",
    "\n",
    "model = tf.matmul(context, out)\n",
    "\n",
    "cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits_v2(logits=model, labels=Y))\n",
    "optimizer = tf.train.AdamOptimizer(0.001).minimize(cost)\n",
    "\n",
    "# Model-Predict\n",
    "hypothesis = tf.nn.softmax(model)\n",
    "predictions = tf.argmax(hypothesis, 1)\n",
    "\n",
    "# Training\n",
    "with tf.Session() as sess:\n",
    "    init = tf.global_variables_initializer()\n",
    "    sess.run(init)\n",
    "    for epoch in range(5000):\n",
    "        _, loss, attention = sess.run([optimizer, cost, soft_attn_weights], feed_dict={X: input_batch, Y: target_batch})\n",
    "        if (epoch + 1)%1000 == 0:\n",
    "            print('Epoch:', '%06d' % (epoch + 1), 'cost =', '{:.6f}'.format(loss))\n",
    "\n",
    "    # Test\n",
    "    test_text = 'sorry hate you'\n",
    "    tests = [np.asarray([word_dict[n] for n in test_text.split()])]\n",
    "\n",
    "    predict = sess.run([predictions], feed_dict={X: tests})\n",
    "    result = predict[0][0]\n",
    "    if result == 0:\n",
    "        print(test_text,\"is Bad Mean...\")\n",
    "    else:\n",
    "        print(test_text,\"is Good Mean!!\")\n",
    "\n",
    "    fig = plt.figure(figsize=(6, 3)) # [batch_size, n_step]\n",
    "    ax = fig.add_subplot(1, 1, 1)\n",
    "    ax.matshow(attention, cmap='viridis')\n",
    "    ax.set_xticklabels([''] + ['first_word', 'second_word', 'third_word'], fontdict={'fontsize': 14}, rotation=90)\n",
    "    ax.set_yticklabels([''] + ['batch_1', 'batch_2', 'batch_3', 'batch_4', 'batch_5', 'batch_6'], fontdict={'fontsize': 14})\n",
    "    plt.show()"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "name": "Bi-LSTM(Attention)-Tensor.ipynb",
   "version": "0.3.2",
   "provenance": [],
   "collapsed_sections": []
  },
  "kernelspec": {
   "name": "python3",
   "display_name": "Python 3"
  },
  "accelerator": "GPU"
 },
 "nbformat": 4,
 "nbformat_minor": 0
}
